# coding=utf-8
import time
from scrapy import Selector
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
import logging as log
from selenium import webdriver
from item import save_motto

index_page_url = "https://www.brainyquote.com/quote_of_the_day"
opt = Options()
opt.add_argument('--headless')
opt.add_argument('--disable-extensions')
opt.add_argument('--disable-gpu')
opt.add_argument('--no-sandbox')
driver = webdriver.Chrome(executable_path='/Users/xiangc/bin/chromedriver', options=opt)
wait = WebDriverWait(driver, 5)

print('start get')
driver.get('https://www.brainyquote.com/')
for i in list(range(50)):
    driver.execute_script("window.scrollBy(0, 400);")
    time.sleep(2)

html = driver.page_source
sel = Selector(text=html)
quotes = sel.xpath("//a[@title='view quote']")
for quote in quotes:
    save_motto(quote, 'all')

# f.write(html)
print('got')
driver.close()
